var citiesList = "wh,xa";

var cities = []
process.argv.forEach(function (val, index, array) {
    if (index >= 2)
        cities.push(val)
});
//var cities = citiesList.split(",");

var path = require('path'), request = require('request'), cheerio = require('cheerio');
var Crawler = require('../../lib/crawler');
//var connection = 'mongodb://localhost:27017/66xiaoqu';
//var connection = 'mongodb://192.168.1.123:27017/66xiaoqu';
var connection = 'mongodb://120.55.126.201/Communities';
var async = require('async');

var MongoClient = require('mongodb').MongoClient, assert = require('assert');

var dbconn = null;
//var collection = null;

cities.forEach(function(city){
//async.eachSeries(cities, function (city) {

    var url = 'http://' + city + '.58.com/xiaoqu/pn_';
    var dbCollection = 'xq_58_' + city;
    var totalPages = 0;
    var numPerPage = 10;

    function getTotalPages(callback) {
        // Parse Total Pages
        request(url + 1, function (err, resp, body) {
            if (!err && resp.statusCode == 200) {
                var $ = cheerio.load(body);
                var totalComm = $("span.tbfilter_l.fl").text().replace(/[^0-9\.]+/g, '');
                var allPages = Math.floor(parseInt(totalComm) / numPerPage) + 1;
                totalPages = allPages;
                // custom added - since 58.com does not go over 250 pages
                //if (totalPages > 250) totalPages = 250
                //
                console.log("Total Pages for " + city.toUpperCase() + ": " + allPages);
            }
            callback();
        });
    };

    function run() {

        var manifest = {
            target: {
                url: url + '{{number}}',
                ranges: {
                    number: [1, totalPages, 1]
                },
                interval: 100 // Wait 0.5 seconds between each access
            },
            plugins: {
                mongodb: {
                    connection: connection
                },
                csv: true
            },

            extractors: [{
                name: '66xiaoqu',
                root: '//tr[@logr="p__8796520510214"]',
                fields: {
                    name: '//td[@class="info"]/ul/li[@class="tli3"]/span[1]/a[1]/text()',
                    addr: '//td[@class="info"]/ul/li[@class="tli2"]/text()',
                    href: '//td[@class="info"]/ul/li[@class="tli1"]/a[@class="t"]/@href',
                    price: '//td[@class="jg"]/p[1]/b[@class="money"]/text()',
                    secondhand: '//td[@class="info"]/ul/li[@class="tli3"]/span[1]/text()',
                    rent: '//td[@class="info"]/ul/li[@class="tli3"]/span[2]/text()',
                }
            }
            ]
        };

        var counter = 1
        function insertDocuments(doc) {
            var name = doc.name.replace('二手房：', '')
            var pinyin = doc.href.split('/')
            pinyin = pinyin[pinyin.length-2]

            var collection = dbconn.collection(dbCollection)

            collection.insert(
                [{
                    name: name,
                    pinyin: pinyin,
                    address: doc.addr.replace('&amp;nbsp;', ' '),
                    price: parseInt(doc.price.replace(/\D/g, '')),
                    href: doc.href,
                    secondhand: parseInt(doc.secondhand),
                    rent: parseInt(doc.rent)
                }]
                , function (err, result) {
                    //assert.equal(err, null);
                    //assert.equal(1, result.result.n);
                    //assert.equal(1, result.ops.length);
                    console.log("Inserted %s (%s) into collection '%s'", name, counter, dbCollection);
                    //callback(result);
                });

            counter += 1
        };

        var job = new Crawler(manifest);

        job.on('data', function (data) {
            //console.log(job._manifest)
            data.data.forEach(
                function (doc) {
                    insertDocuments(doc);
                })
        });

        job.on('error', function (err) {
            throw err;
        });

        job.on('end', function () {
            console.log('The job is done');
            // not exactly,
        });

        function init(callback) {
            // Connect to MongoDB
            MongoClient.connect(connection,
                function (err, db) {
                    assert.equal(null, err);
                    console.log("Connected correctly to server");
                    dbconn = db;
                    //collection = dbconn.collection(dbCollection)
                    callback(null, 'init');
                });
        }

        function start(callback) {
            console.log("starting...");
            job.start();
            callback(null, 'start');
        }

        function finish(callback) {
            console.log("closing db...");
//    dbconn.close();
            callback(null, 'end');
        }


// Start our job

        async.series([
                //getTotalPages,
                init,
                start,
                finish
            ]
            , function (err, results) {
                console.log(results);
                console.log('END');
            });
    }

    // Main method
    getTotalPages(run);
    //run()
});
